{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation - IJGI_2018\n",
    "\n",
    "For http://www.mdpi.com/journal/ijgi Journal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluations and results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import sklearn.metrics as skmetrics\n",
    "from os.path import join\n",
    "\n",
    "#path=\"/media/data/marc/eval/fields/convgru128\"\n",
    "#path=\"/media/data/marc/eval/2018_IJGI/convlstm256_48px/eval\"\n",
    "\n",
    "datafolder=\"data_IJGI18\"\n",
    "model24px=\"convgru256\"\n",
    "model48px=\"convgru256_48px\"\n",
    "\n",
    "write_flag=False\n",
    "\n",
    "#path=\"data/bavaria/models/fine_tuned/convgru256/eval\"\n",
    "quantevalpath=join(datafolder,\"eval\",\"full\",model24px) # 'full' or 'demo'\n",
    "qualevalpath=join(datafolder,\"eval\",\"demo\",model48px)  # 'full' or 'demo'\n",
    "\n",
    "docpath=join(datafolder,\"doc\")\n",
    "\n",
    "dat = np.loadtxt(join(quantevalpath,\"2016\",\"classes.csv\"),dtype=str, delimiter=',')\n",
    "ids = [int(id) for id in dat[:,0]]\n",
    "names = classes = dat[:,1]\n",
    "\n",
    "colarray = [[0,101,189],[0, 0, 0],[79,76,76],[0,82,147],[152,198,234],[100,160,200],[218,215,203],[162,173,0],[227,114,34],[105,8,90],[15,27,95],[0,119,138],[0,124,48],[103,154,29],[255,220,0],[249,186,0],[214,76,19],[196,72,27],[156,13,22]]\n",
    "tumcolors=np.array([np.array(col+[255],dtype=float)/255 for col in colarray*10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Satellite Availability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.5, 1.2)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzkAAAB0CAYAAACmPPw2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztnXucXVV597+/uWSSmczkAiSQBAi3\nhJsQAcH4ggKCaAUBbQVsgVQtVFut9dL6Wt+qbfVjVWy9YC2liqIgICpGlEsNEdAUiJE7JEQu5kJI\nApnMZJKZTGae94+1z7DnzNnn7HPOnjnDnOf7+cxnzjlr7ee21n72XnutvbfMDMdxHMdxHMdxnIlC\nQ60NcBzHcRzHcRzHyRIf5DiO4ziO4ziOM6HwQY7jOI7jOI7jOBMKH+Q4juM4juM4jjOh8EGO4ziO\n4ziO4zgTCh/kOI7jOI7jOI4zofBBjuM4rzgk/amkOzKSZZIOzUJWLZG0XNJ7x0DP/ChmTaOtK0uy\n7DNObZB0jaR/qbUdjuO8MvBBjuPUMZKOkLRM0nZJayWdn1f+TklPSOqW9Lik84rImifpZklbI3mP\nSlqSgY0jTqrN7Ptm9qZqZafQ/SVJT0X+PynpkhTbHCRpUNJ/jLZ9aZH0aUn9knZEf09Ieket7Rot\natlnxgOSXivpTkkvSdoi6SZJ+8XKJelfJb0Y/f2rJMXKr5K0OurHSwrIP1jSz6L9YqukL4yRa4lI\nequkeyV1Stok6WpJ7bHyFknfktQVlX84VjZJ0g8lPRv1m1PzZP8itu/skLRb0iNj6J7jOBXggxzH\nqVOiE8BbgJ8BM4HLgO9JWhCVzwW+B3wY6AA+BlwnaVaCyGuBdcCBwF7AxcALo+nDGNADnANMAy4F\nviLpdSW2uQTYBlwgqWWU7SuHG8xsqplNBT5EaOvZtTaqENFJuB+fUpAwozYDuAqYT9gfu4Fvx8ov\nA84DjgWOIfTxy2PlDwHvB1YV0DcJuBNYBuwLzCPkiVozDfgXYA5wBDAX+GKs/NPAYYR4nAb8naQ3\nx8rvBf4M2JQv2Mzektt3ov3nN8BNo+GE4zjZ4QcRx6lfDiecEPybmQ2Y2TLg14TBCYSTl04z+4UF\nbiWc9B+SIO81wDVm1mNme8zsd2b2i1xhdHX5N9GV1ofiV0ujpVb/LOnX0dXhOyTtHRXfHf3vjK6i\nLpa0RNK9se1N0l9Gsy6dkq7MuzL97mj2Ypuk2yUdmCZAZvYpM3vSzAbN7D7gHmBxUv1I5yXAJ4F+\nwsljItEV9k3RzNfdko6KlV0T+XFrFJP7JB0SKz8zml3aLunrgAoqKezX7YQT34JtKalB0iclPSdp\ns6TvSpqWV+3dkjZKel7SR2PbnihpZXTF/AVJX46VleoDn5X0a2An8DFJK/Ps+ltJP40+v1XS7yI9\n6yR9OlY1TZ95naQHovg9EB+8FuuPkiZL+p7CDEhntG3BwaLCTOnyqN5jkt4W/X5S1O6NsbrnS3o4\nFv+PS/p9pOdGSTOjstws1Xsk/YEw2BhGtM/eZGZdZrYT+Drwf2JVLgWuMLP1ZrYBuAJYEtv+SjP7\nJdBbwK0lwEYz+3K0r/ea2cOF/I/s/UrUPl2SfivplFjZpyPfvhvF+TFJJ8TKXy1pVVR2AzA5SY+Z\nXWdmt5nZTjPbBvxXAZ//2cy2mdkTUfmSaNvdZvbvZnYvMJCkI7JpPnAK8N1i9RzHqT0+yHEcJ46A\no6PPK4EnJL1NUqPCUrU+IOmE5n+BKyVdKOmAYULDrNCthCutM4GPAjdL2idW7V3AnwOzgElRHYDX\nR/+nR1dSVyToP5sw0DoGeCdwVqT7XOATwNuBfQgDleuLRqEAkqZE8h8rUu1kwuDwB8CNhBOrYvyC\ncHV5FuGq+ffzyi8EPkO4Mr8W+Gxky97AjwiDqb2B3zP8hK6YH5L0VkKMH0+otiT6Ow04GJhKOFGO\nc1pk+5uAv5d0RvT7V4CvmFkHYRB1Y6Q3TR+4mDDL0A58E1go6bBY+buA66LPPYQB5XTgrcD79PJy\nyqJ9Jhow3Ap8lTDr+GXgVkl75ekq1B8vJcwa7B9t+5fArrzYIKkZWArcEcn4APB9SQujAXMPcHqC\nbx8gzLS8gXAhYhtwZZ6KNxBmLM7K112A1zO83x5FmK3J8VD0WxpeCzyrsIRrazSIe1WR+g8Aiwht\nfh1wk6T4YOVthP1lOvBTon6mMGP0E8IM8UzCzEk5SyyHfJY0A9iPyn2Ocwlwj5k9W8G2juOMJWbm\nf/7nf3X4BzQDTwN/F31+E7AbuD1W5z3ADmAP4er6W4vImwF8nnBiMQA8CLwmKvt74Nq8+rcDl0af\nlwOfjJW9H7gt+jwfMKApVr4EuDf23YCTY99vBD4eff4F8J5YWUPky4GxbQ9NEa/vALcBKlLnauAn\n0efFhNmcWSnbY3pky7To+zXA1bHyPwKejD5fAvxvrEzAeuC9CbI/HbVtJ+HkegD4uyK2/BJ4f+z7\nwsiXplh7HB4r/wLw39HnuwkDs73zZKbpA/+UV/494B+jz4cRZp9aE2z+d8KsZMk+QxhM3Z+3/Qpg\nSYr++G7CcqVjSrTnKYSlTw2x364HPh19/hfgW9Hn9qhdcn3yCeCNse32KxD/g1P2q2OAl4BTYr8N\n5LXfYZFM5W17by4msd/uiGx5C2Hw9zFCHpmU0p5twLGxfvk/sbIjgV3R59cDG+M2RXH/lxQ6zoz0\nLIi+7x/5NzmvzrMFtl0PnFpE9tr8mPif//nf+PzzmRzHqVPMrJ9wtfithJOxjxAGB+sBoivzXwBO\nJZzMvAG4WtKiBHnbzOzjZnYUMJswyPmJJBHWwf9JtGynU1InYdZjv5iI+Fr4nYTZg3JI2v5Awr00\nOb0vEQYFc9MKlvRFwgzXO83MEupMAf6EaDbGwuzBHwhX6AvVb5T0+WhJUhfwbFS0d6xakk9zCPc/\nEemy+PcEbjSz6WbWRphhuUTS5Ql15wDPxb4/RzjBji/LWpdXPif6/B5gAfBktJTr7Oj3NH0g34fr\ngIuiz+8iDCB3wtCSr7sUbqzfTphR2Zt05PuX8yHeJ5Jify1hcPYDheV6X4hmbQrpWGdmgwk6rgPe\nrnDf1tuBVWaWs+lA4MexOD1BGJgkxb8gCk8N/AXwN2Z2T6xoB+E+uxwdwI6kvp3HLsJg8Rdmthv4\nEmFG64gEGz6qsFR0e+TLNIr38ckK9xnNATbk2ZTfZoX0vZYQ2z82szXRzzui//k+d5eSlyf7ZMJ9\nSD8sZzvHcWqDD3Icp44xs4fN7A1mtpeZnUVYmnR/VLwIuNvMVlq4J+UB4D7gjCR5MblbCSc/cwhL\nTdYRruJPj/21mdnn05hZiW8x1gGX5+meYma/SbOxpM8Qrlq/ycy6ilQ9n3Di9I3ofotNhBPapCVr\n7wLOJcRzGuEKPaS7t+Z5wtXpnI2Kfy+FhaU2vyD5nqGNhBPtHAcQZvPiD5LYP698YyT7KTO7iLBE\n61+BH0pqI10fyG/rO4F9ooH1Rby8nIvo80+B/c1sGmF5Wy52pfpMvn85HzaU2A4z6zezz5jZkcDr\nCMskCz11byOwv4Y/QGFIh5k9TjhpfwvDl6pBiNVb8mI12cL9M0OmFLNT4b6z/yHch3JtXvFjhIcO\n5DiW4ssw4zxcSnfMhlMIM8XvBGaY2XRgO+n7+Nyob+c4IKlypO/VhD7xbgv3FAHhAkwkr1Kfc1wK\n/MjMdpSs6ThOzfFBjuPUMZKOiW6kblW4eXw/wjIpCGvpT8nN3EQnEKeQcE+OwmNoj5bUpPDo1vcB\na83sRcKyo3MknRXNYEyWdKqkeSnM3AIMEgZglfBN4P8quqlf0jRJf5JmQ0n/l3ACekbkRzEuBb4F\nvIowQFxEuE/m2IR7FtoJ9zi9CLQCn0tjU8StwFGS3h5d9f4g4QpzKqK4v5nkk7zrgb9VeBz21Mi2\nG8xsT6zO/4v6zVGEe1duiGT/maR9ohmMzqjuIBX0gWi28SbCU7JmEgY9OdqBl8ysV9KJDJ8xK9Vn\nfg4skPSuqL9eQFgq9bMkW3JIOk3SqxQeGtBFWLo1WKDqfYSZib+T1KzwkIVzCPef5LgO+BvC0qz4\n07q+CXw2GqggaZ/o3rJURPc/LQO+bmbfLFDlu8CHJc2VNIcwi3tNbPtJ0X0zApqjtsqdL3wPeK2k\nM6IYfAjYSphtyqedMDjeAjRJ+keGz6YUY0W07Qej+L0dOLGIz0cTlpN+wMyWJvj8SUkzJB0O/EWe\nzy2xe4UmRT7HH14yhTBYuwbHcV4R+CDHceqbiwlXODcDbwTONLM+ADP7FWHN/A8ldQM3A58zs6QX\nKrYCPyac2D5NuFL+tkjWOsKsxScIJzzrCGv5S+agaHnSZ4FfR8t3XluOg2b2Y8KMwg+iZWGPEq6e\np+FzhKvHa/XyOzI+kV8pOql8I/DvZrYp9vdbwolXodmc7xKu5G8gPADgf8vwaSthadznCYOkwwhP\nxivGBTkfCAPYXxPunSnEtwjLsu4GniE8ZesDeXV+Rbg/4ZfAl2L94s3AY5GerwAXmtmuKvrAdYTZ\nrpvyBlnvB/4p6pv/SPSAAyjdZ6IB69mEk/sXCbMNZ0dxLUVuuVIX4cT+V4RYDSNaynUOoa9tBb4B\nXGJmT8aqXU9YBrosT/dXCDMSd0T+/S9wUgrbcryXMMD7dKzfxmcf/pPwUIRHCPvDrdFvOe4gLEt7\nHeFR1LuIHuZgZqsJj1r+JuG+l3OBt0X+5nM7of+vIfT1XlIss4v07CYs41tCWGJ6AeFhG0l8hPBg\nkf+O+RwfxH+K8ICO5wht9kUzuy1Wvjryc25k9y6Gz/adR8htd6Wx33Gc2qN0S3Adx3Ecx3Ecx3Fe\nGfhMjuM4juM4juM4Ewof5DiO4ziO4ziOM6HwQY7jOI7jOI7jOBMKH+Q4juM4juM4jjOhGDeDnBUr\nVhjh2fs1+aul/lrp9pjXn+/16vd40F+vvter37XUX48+17PftdZdz/rr1e9a6yYl42aQ09fXV7f6\na6XbY147vM3rT3+9+l6vftdSfz36XEu940F/PfteS/316netdaelqdYGOI7jOI7jOJXR1dVFf38/\nXV1ddHR0jCjr7u6mvb19RFmxeuVuNzg48n24aWWUs028HBj2uVrdae0qprcc3UkyK23DSv3JSuZ4\nxAc5juM4juM4rzDWb9jILctWsLmvkYUzGrjixmXMahng3NMXAwyVWXMr6t85VDZv7pyCMqy5le4t\nG3h+3TPMmb+AqTNnpd7u8LadXHntzWXpTvKl0Dbx8u09u1i7ejUNLa0ccuhhdLSIo2cGGZXoThvj\nYnrzbUwb80Iyy23DSv3JSuZ4JtUgR9I/AO8CBoBB4HLgg8AJQD9wP3C5mfVLWgJ8kfAW72bCG6Ev\nid5A7TiO4ziO41TB+g0buWrpPUw/eBHTJZp6n2H6gQfRZ8aXrl1KQ2MT8449menS0DZ9Zly19B4u\nO+eUoZPyuIzOrZtZvbWXtsPPYm3nZo5v34uO9vkltwNo6n2Gvo75qXUX8yV/m3Nes4ClD6xh+sGL\n4MUtrFn7CFOPeRMAv9/2Ascv3J/Bpq0V6U4b42J6821MG/MkmX1Tp2buS5o4VyJzvFPynhxJi4Gz\ngePM7BjgDGAd8H3gcOBVwBTgvbHNbjCzRWZ2FLAbuCBrwx3HcRzHceqRW5atYPrBi1DsZBVAEht2\nDPK89ipYNv3gRdxy14qCMh5/7BGmzj+WhoYGJs+Yzepn1qfarhLdaX2ZfvAirrj2x0PlORslIWnI\nTomKdBcjblcxvfk2po15sszK4liOP1nJHO+kefDAfsBWM+sDMLOtZrbRzH5uEYSZnHn5G0pqAtqA\nbVka7TiO4ziOU490dXWxua9xxMkqQG/PDnYymZ2DDQVvDJfE5r5G1q9fP0xGbrvcd0n0DGhIRtJ2\nleju6upK5QtA384etloHu3fvHmFj3M6BPXvK1l2MuF3F9A4ODg6zsRCFYldM5vZtL2bqS74/xWws\nR+YrgTSDnDuA/SWtkfQNSW+IF0pqBi4Gbov9fIGkBwlL1mYCSwsJlnSZpJWSVi5dWrCK4ziO4ziO\nE9Hd3Y01txYs693VAy1tWEMzu3cXfvqVNbexadOmYTJy2w2rlyej0HaV6O7u7k7ly5DMKdPZvbuv\noI05OwcGBsrWXYy4XcX0mtkwG5PIj10xmTu2b8vUl3x/itlYjsxXAiUHOWa2AzgeuAzYAtwQ3XeT\n4xvA3WZ2T+y3G8xsEbAv8AjwsQTZV5nZCWZ2wjnnnFOhC47jOI7jOPVBe3s76i98m/PkKW3Q14MG\n+5k0qaVgHfX3sO+++w6TkdtuWL08GYW2q0R3/KlkxXwZkrmrk0mTWgramLOzsbGxbN3FiNtVTK+k\nYTYmkR+7YjKnTpuRqS/5/hSzsRyZrwRSvSfHzAbMbLmZfQr4a+AdAJI+BewDfDhhOyPM4rw+G3Md\nx3Ecx3Hql46ODma1DBBOsYYzuW0qrfTS2jBIS8vIE2QzY1bLAPPmzRsmI7dd7ruZ0dZoQzKStqtE\nd/yRxcV8AWhpbWNvdTFp0qQRNsbtbGxqKlt3MeJ2FdPb0NAwzMZCFIpdMZnTZuyVqS/5/hSzcaI9\nTjrNgwcWSjos9tMi4DlJ7wXOAi4ys5EPSH+Zk4HfV2em4ziO4ziOA3Du6YvpfPrBESetZsbcqQ3s\nZy8WLOt8+kHOPW1xQRlHHvUqdjz7EIODg/Rue4GFB81LtV0lutP60vn0g3zk4vOHynM2mllYKhbZ\naUZFuosRt6uY3nwb08Y8WWZlcSzHn6xkjnfSPEJ6KvA1SdOBPcBawtK1TcBzwIroRqYfmdk/Rdtc\nIOlkwiBqPbAkY7sdx3Ecx3Hqknlz53DZOaeEd57sbmT29EY6Nz/OrJYBPnpxWP6fK7PmNtTfw6yW\ngWGPCc6XYc1tLNxnMs+vvp1D5y9gYMdLdG5bl2q72a27aNn+aGrdxXwptM1++87mlmUrYHcjC/ab\nwdpH7qChpZVDDzmMwW3raJhBRbrTxriY3nwb08Y8SWbLzvRtWE2fyULmeKfkIMfMfgu8Lu22ZnYN\ncE1VVjmO4ziO4ziJzJs7h7+6+B10dXVx3333cfaZJw1bbpQrK/Zm+7iMUC/IKHe71atX867TTy9L\nd3E7hm8zsvyPAIbqrlq1augkvVzd5dmVrLeUD2llltuG1flTvczxTKqXgTqO4ziO4zjjj46ODpqb\nmwuerHZ0dKQ6ic2vV+52Tz31VMW6y9mmkJ2VyCmXtHrL0V0rX0ZL5ngk1YMHHMdxHMdxHMdxXin4\nIMdxHMdxHMdxnAmFD3Icx3Ecx3Ecx5lQ+CDHcRzHcRzHcZwJhQ9yHMdxHMdxHMeZUPggx3Ecx3Ec\nx3GcCYUPchzHcRzHcRzHmVD4IMdxHMdxHMdxnAmFD3Icx3Ecx3Ecx5lQ+CDHcRzHcRzHcZwJhQ9y\nHMdxHMdxHMeZUPggx3Ecx3Ecx3GcCYUPchzHcRzHcRzHmVD4IMdxHMdxHMdxnAmFD3Icx3Ecx3Ec\nx5lQ+CDHcRzHcRzHcZwJhQ9yHMdxHMdxHMeZUPggx3Ecx3Ecx3GcCYUPchzHcRzHcRzHmVD4IMdx\nHMdxHMdxnAlFU60NGE90dXXR3d1Ne3s7HR0dFdXr6upi48aNAMyZM6dgeRod1dqYFZXEBEhtY7Ux\nL0dvvO7g4CAbNmxIVTdNnEv1ibGKR6l6pfxOIyNtzHP1zAxJQ/Ur9SW/bjHdSXUHBwfL9jGpvyXZ\nmmRjWr/SyM2qH5WqX8l+XY4v8b4xFvlsNMkiN1fav8cydmn3ldGwd7RzKVTfx0crn5Wru5i8So/X\nxYgfW7L2Jat+VsvzmdFsm1rlglcCPsgB1m/YyJaXOrnixmVYcyvq38mslgHOPX0x8+bOGVbvlmUr\n2NzXOKIewHdu/jkrnlxP76TpqLmFKXu6OemQvbn0/DcDFN02jY1J28dtzDImafTF623v2cXa1atp\naGnlkEMPo6NFiTZWG/MTjzyY+x9/OpXeQjae97ojWPrE9qJ108S5VJ9I2+aVxLucet1bNvD8umc4\n+/8sYvm6x1L1nyQZc+YvYLChMTHmOb/XvtDFmuc20qcWWtumcOisaSw+cCrrN2wsO9bl9LNidV8z\nW1x57c2J7R33cerMWQX7W5KtpWw8eiZDvpfTnpXGKU2eyWL/KiVrwb4dI+oU6huH7D1l1PLZaJJF\nbq60f4/FsSDJz6R9Jb+vZR2favaBao9dpfavNMe00dZdTB5Qtt60bTNjSiM//eHdmfiStp/FZfb3\n93PltTeXfSzO6nwmjT9Zt03a86h6RmZWupL0D8C7gAFgELgc+CBwAtAP3A9cbmb9sW1+AuxrZq9N\nY8jy5cvt1FNPLdf+qlm/YSNXLb2Ho+ZMZ+OUg4Z+NzM6n36Qy845ZejActXSe5h+8CIkDav3h1XL\n6enpYcPuKUw96NihcjNj10ub2HfXs7R3TGPesSeP2Lbz6Qc55ZCZnHnmmSVtLKQ7bmO5LF++nEIx\nT6svXm/7i1t44KFHmDr/WAB6t73A8QsPoH3q1BE2VhvzbVte4Ne/WsYpb3ortru3qN5zXrOApQ+s\nGWHjm/faxk/W7k6smybOpfpEQ2NTqjavJN7l1OvcupkHHnqEtgOP4awZW9gwaX862tuL9p9iMjr/\nsBrr62bmgteMiHnO76lzF7Dy4UeZOj/sD2ZG77YXuOiIFp7Y1F1WrMvpZ0ntnat70REtPN96SMH2\njvvY17mZ4xceQEd7+7D+Ni12lSxuK1DSxsObtvLYxs6y2jMut5w4pckzWexfaXzZd/vjvP51Jw35\nQsecgn3juAX7M7jl9xXnsySS8lwWpGnDtU+tKaq/0v5drD9k7XOxfBDfV/L7WhbHrnJk3Hnnndzz\n+5dG5diVJt+ftHBe0WNavj9Z6368py1RXvyYVM7xOm3b7LvjKb53/7pMfMkdN0v1s7jMu39zH5um\nHVn2sbja85lifS5tzCtpm7TnUaPJaObWFKh0lRT35EhaDJwNHGdmxwBnAOuA7wOHA68CpgDvjW0z\nHTgemCbp4LJNH0NuWbYi6pzDf5fE9IMXcctdK/LqaUS9DTsGeXhD17ABTq5sysx9eXhDF89rr4Lb\nTj94EZ3dPSltLLx9zsasSKsvXu/xxx4ZOmmRxOQZs1n9zPqCNlYb8ycef5SZi85gzbMbSuq94tof\nF7QRUbRuMbvSxGnDjsHUbV5JvMupl/O7oaGBhsZmVj+zvqhfpWR0btvGrmkHFYx5zu8nHn/5JDan\na/KM2fT29Zcd63L6WVJ7x/UntXfcx5xcGN7fkmxNZyNlt2c5+36puvl5Jov9K40vjS2tw3xJ6htr\nnt0wKvlsNMkiN1favyvVVwnF8kF8X8nva1nHp5SMzu6eUTt2pcv3FNU72rqLyYsfk8o5Xqdtm97e\nXRn6kq6fxWU2trRWdCyu9nymWJ9LG/NK2ibteVS9k+bBA/sBW82sD8DMtprZRjP7uUUQZnLmxbZ5\nO7AU+AFwYdZGZ0VXVxeb+xpHdM4cktjc18j69esT6/X27GB7n7GrqZ2BgYER5Xt6d7KrqZ3u3UZf\nX19BHf2DwZZqbEzavlwqiUlvzw52MnnYNpLoGRB9fX3DbKw25jldDQ0NdPbsonugOVFv384etloH\nu3fvLmljvG4xu3JxLuZHTtfOwYaSbZ5FHyxWb4TfYsjnQn4V8i0uo39XD3uap7DHxJ49e4bFcfu2\nF9nJZLbv7B3RLrl6g0BX50upYx23pZw2TKo7CAXbO79+vk8NDQ3D4haXua6zj3XdAyVtHBwcLKs9\n43LLiVNS3XieSapfzv6V3heGfOnb2TMiPnG5u3fvzjSfjSZp992k+8DyZWSdo7KiWD7ItzH3vX+Q\n1DmrmL3lHP+6urroH2RUjl2lzgFy+b5QW1eaz8rVPbBnT6K8XB7bOdgw9DnN8boY+b4M0pCZL319\nfan62XCZI21Mcyyu9nwmqc/FdReLeSVtk8W+VS+kGeTcAewvaY2kb0h6Q7xQUjNwMXBb7OeLgOuj\nv4uSBEu6TNJKSSuXLl1avvVV0t3djTW3Fq1jzW1s2rQpsV7vrh4GaIDJHQUHOf29O2FyB3toZPfu\nkTsZAGqku7u7KhuTti+XSmLSu6sHWtpG1mtoHvI5Z2O1MY/r6u/vZ6C5JVFv764emDL95c9FbIzX\nLWZXLs7F/Mjpivs/gqjNs+iDxeoV8jvfrvz+k2/TsJj37qShpQ1Tw7D+bg3N7Ni+DVraEtsFwFCo\nlzLWcVvKacPEuqhgeyfFKedTobgNxWeP2G1NJW3MLQ1O255xuUmUs1/F80xS/XL2r0p8SYpPXG6W\n+Ww0SbvvFjouFJKRdY7KimL5IN/GIdSYOmcVs7ec4193dzeo+ClNpceuUucAuXyftPy/knxWru6B\ngYFEebk8lp/Tiukt1Y9G+FIg9pX6kpTDCx27ypFZiGrPZ5L6XFx3sZhX0jZZ7Fv1QslBjpntICw9\nuwzYAtwgaUmsyjeAu83sHgBJs4HDgHvNbA3QL+noBNlXmdkJZnbCOeecU50nFdDe3o76dxato/4e\n9t1338R6k6e00cgg9HbR2Ng4orx5civ0dtHEAJMmFT7xwwYSn76U1sZST29KSyUxmTylDfpGLrnT\nYP+Qzzkbq415XFdzczON/QWuzkR6J09pg12dL38uYmO8bjG7cnEu5kdOV9z/EURtnkUfLFavkN/5\nduX3n3ybhsV8ciuDfT3IBof1dw32M3XaDOjrSWwXAGGhXspYx20ppw0T62IF2zspTjmfCsVtKD5N\nxiTtKWnj0BXJlO0Zl5tEOftVPM8k1S9n/6rEl6T4xOVmmc9Gk7T7bqHjQiEZWeeorCiWD/JtHMIG\nUuesYvaWc/xrb28HS541y9Wt5NhV6hwgl+8Tr6xXkM/K1d3Y2JgoL5fH8nNaMb2l+tEIXwrEvlJf\nknJ4oWNXOTILUe35TFKfi+suFvNK2iaLfateSPWeHDMbMLPlZvYp4K+BdwBI+hSwD/DhWPV3AjOA\nZyQ9C8ynyGxOLeno6GBWy0Dy1RczZrUMMG/evMR6k9umMq1FTNnTXfBg1jS5lSl7ummfJFpaClwV\nNaO5gcTH/qW1MavHBlYSk8kXVtEKAAASJ0lEQVRtU2mld9g2ZkZbo9HS0jLMxmpjntM1ODjI9LYp\ntDf2J+ptaW1jb3UxadKkkjbG6xazKxfnYn7kdLU2DJZs8yz6YLF6I/w2hnwu5Fch3+Iymqe00dS/\niyYZTU1Nw+I4bcZetNLLtNbJI9olV68B6Jg+M3Ws47aU04ZJdRugYHvn18/3aXBwcFjc4jL3n97C\n/u2NJW1saGgoqz3jcsuJU1LdeJ5Jql/O/pXeF4Z8aWltGxGfuNxJkyZlms9Gk7T7bkND8mG20v5d\nTF/WsSuWD/JtzH1vbiB1zipmbznHv46ODpobGJVjV6lzgFy+L9TWleazcnU3NjUlysvlsdaGwaHP\naY7Xxcj3pYHBzHxpaWlJ1c+GyxxpY5pjcbXnM0l9Lq67WMwraZss9q16Ic2DBxZKOiz20yLgOUnv\nBc4CLjIbNpS9CHizmc03s/mEWaBxe1/OuacvpvPpB0fsIBY9oeLc0xbn1Rt5cJ47tYFj5naw45mH\nRnTOXS9t4pi5HexnLxbctvPpB5neXnj5xkgbC2+fszEr0uqL1zvyqFex49ngv1l4WtLCg+YVtLHa\nmB9x5NG89OD/sGD+3JJ6P3Lx+QVtxChat5hdaeI0d2pD6javJN7l1Mv5PTg4yOBAPwsPmlfUr1Iy\nps+YwZTtzxSMec7vI448+uVYR7p6t73A5JbmsmNdTj9Lau+4/qT2jvuYkwvD+1uSrelspOz2LGff\nL1U3P89ksX+l8WWgb+cwX5L6xoL5c0cln40mWeTmSvt3pfoqoVg+iO8r+X0t6/iUkjG9vW3Ujl3p\n8j1F9Y627mLy4sekco7Xadtm8uQpGfqSrp/FZQ707azoWFzt+UyxPpc25pW0TdrzqHqn5COkJR0P\nfA2YDuwB1hKWrm0CngNyi/5+BHwX+DUwz2KCJa0C3mdm9yXpqdUjpCE8BnHFfffz6LawjlH9PeFZ\n46clPEN9d+OIehC9J2f1enonzUDNk6L35OzDpeedBZC4balHjJbSXeljAos9/i+tvni97Tt6Wbvm\nSRpaWjn0kMNon6xEG6uN+YlHRO/xSKG3kI3nLT6Cx7dPKlo3TZxL9Ym0bV5JvMup171lfXhPzusW\nsaa3PVX/SZIxZ/4CBtWUGPOc32s3d7Hmuefp0yTaWqdwyOxpLD5gKotPOrHsWJfTz4rVPWG22Nab\n3N5xH6fOnF2wvyXZWsrGo2cw5Hs57VlpnNLkmSz2r1KyFszuGPa49KS+ccjeU6rKZ0mM9mNOS7VP\nGv2V9u+k/jAaPqfdV/L7WhbHrrQyli9fzqGHLRi1Y1epfJ/mmDbauovJA8rWm7ZtZkxp5P4N2fiS\ntp/FZd55552s2dRV9rG42j5Rqs+ljXklbZP2PGq0eCU8QjrVe3LGgloOciL9HHfccZm8UXfjxo0A\nzJkzJ9Ubd8vpKFm+2TaN3tF8Q3AWMa/0DeErV65k4cKFmb1NvNy3LCfFPuu3h+fXW7ZsWVG/08go\n983NZi+/1X7VqlUj/B6tN4QXqrt69WpOP/30snxM6m9JtibZWMj3SuVW8ubtYvt7FvtXkqxCegv1\njdFaWjFWB+KkGFaa3yF9/84vH02f0+wrxdq8mrYuJSOudzSPXcXaOu0xbbR1F5NXid5SxI8t1fpS\nqCxt25d7LC5lVylf0va5co+baeNYbp/LklfCIKf4o3vqjNway2rqlZKRVsdobT9a+vLrpbWx2piX\nozdet6Ghgblz56aqm4Zq+kQldSutV8rvNDLSxjxrX8rRnVT3qaeeSl03TX+r1sZq5GZVt1j98eDL\nK4UsfKqmf48VafeVUttloTuLuln38dHKZ5XorrZeOeQfW0bDl7Fqm9E4nxnNtpmI+TQrUj14wHEc\nx3Ecx3Ec55WCD3Icx3Ecx3Ecx5lQ+CDHcRzHcRzHcZwJhQ9yHMdxHMdxHMeZUPggx3Ecx3Ecx3Gc\nCYUPchzHcRzHcRzHmVD4IMdxHMdxHMdxnAmFD3Icx3Ecx3Ecx5lQ+CDHcRzHcRzHcZwJhcys1jYA\nIOlRoLeGJhwA/KFGuvcGttaR3vGgv159r1e/x4P+evW91n7XMrdDbfyvdczrta95fqk//fXqd611\nbzWzN5eqNJ4GOSvN7IQa6t9iZvvUSHdNfB8HMa+Z/nr1vV79Hg/669X3ceB3zXJ7pH/M/R8HMa/X\nvub5pc7016vftdadFl+u9jKdtTbAcRzHyRzP7Y7jOHWID3JeZnutDXAcx3Eyx3O74zhOHTKeBjlX\n1bH+Wun2mNef/nr1ezzor1ff69XvWuqvR59rqXc86K9n32upv179rrXuVIybe3Icx3Ecx3Ecx3Gy\nYDzN5DiO4ziO4ziO41SND3Icx3Ecx3Ecx5lQjMkgR9KApAclPSbpIUkfkTTmAyxJO8ZaZ6Q353/u\nb36RuqdK+llGek3S92LfmyRtyUp+ShvOi+w4fIz01dznmO6a9LdybJC0XFJmj4Ac6/YuoP8fojzz\ncLSvnTTG+udJukXSU5J+L+krkiYVqf8hSa0Z6DVJV8S+f1TSp6uVm1K35/cxzO/jJcd5bq8dY53X\nI5l1m9trldcjWTXJ7eMlr1fLWBm8y8wWmdlRwJnAW4BPjZHu8UDO/9zfs2Oktwc4WtKU6PuZwIZy\nBEhqqtKGi4B7o//l6G2sUF/VPjtVUVF7Z4GkxcDZwHFmdgxwBrBuDPUL+BHwEzM7DFgATAU+W2Sz\nDwFZHAz7gLdL2jsDWeXi+X1s8/t4yOvgub3eqMvcXuO8DrXL7RMir4/5qMzMNgOXAX+tQKOkL0p6\nIBqhX56rK+nvJT0SjSI/n4V+SVMl/VLSqkj2udHv8yU9Iem/opHrHbFkmjnF/AY6JN0qabWkb1Y5\nev458Nbo80XA9TEbTpS0QtLvJP1G0sLo9yWSfippGfDLShVLmgqcDLwHuDD67VRJdxfyT9IOSVdI\neghYXKleKvP5bkmLYvXulXRsFTbk5Ay7civp65KWRJ+flfSZWF8clStkxWzIWE9Seyf5/0eSnpT0\nW0lfVfVXZPcjvAW5D8DMtprZRknHS/pVpOd2SftF+pdHV+QelPSopBOr1H860Gtm3470DwB/C7xb\nUpukL0V6Hpb0AUkfBOYAd0m6q0rdewhPuvnb/IIoty2L9P5S0gGSpkl6LrbvtUlaJ6m5GiM8vw/Z\nMdr5vWZ5PZLlub3GuX2s8noku55zey3zOoyD3F7rvF4NNZl6MrOngUZgFmGn2W5mrwFeA/yFpIMk\nvQU4FzjJzI4FvpCR+l7gfDM7DjgNuEKSorLDgCujkWsn8I6MdE7Ry0sZfhz9VtDvqOxE4APAkcAh\nwNur0P0D4EJJk4FjgPtiZU8Cp5jZq4F/BD4XKzsO+GMze0MVus8FbjOzNcCLko6Pfk/yrw24z8yO\nNbN7q9Bbic//DSwBkLQAmGxmD1VhQ1q2Rn3xP4CPjoG+0SSpvUcQtc1/Am8xs+OBLN5Ifwewv6Q1\nkr4h6Q1RYv8aoS8fD3yL4VfgWs1sEfD+qKwajgJ+G//BzLqAPwDvBeYDi6Irkd83s68CG4HTzOy0\nKnUDXAn8qaRpeb9/DfhOTi/wVTPbDjwI5Pbvs4Hbzay/WiM8vwOjn99rmdfBc3saPLdPjNxe67wO\n4yC31zivV8x4WF/3JuASSQ8SEtZehIPRGcC3zWwngJm9lJE+AZ+T9DDwP8BcYHZU9oyZPRh9/i2h\n82ZBfDnD+dFvSX4D3G9mT0dXDK4nXEGpCDN7mODHRYSrYHGmATdJehT4N8LOnOPODGJ+EeGgRPQ/\nN82d5N8AcHOVOiv1+Sbg7Chxvhu4plo7UvKj6H+W/a1WJLV3IQ4HnjazZ6Lv1xepmwoz2wEcT7ji\ntAW4AbgcOBq4M9rXPgnMi212fbTt3YQr7NOrtSOBU4H/NLM9kb6s8tkQ0YH3u8AH84oWA9dFn6/l\n5f3tBuCC6POF0fes8fw+Cvm9xnkdPLenwXN7YCLn9lMZ5bweyR1vuX2s83rFZLEut2wkHUxIepsJ\nB6UPmNnteXXOGiX1f0q4snC8mfVLehaYHJX1xeoNAKO2nIFkv08F8l9eVO3LjH4KfImwQ+4V+/2f\ngbvM7HyFm2WXx8p6qlEoaSZhmvdVkoxwBcCAW0n2rzc6OGZBWT6b2U5JdxKuQryTkFCzYA/DLyZM\nzivP9bkBRm9/LGVD1RRp71tGW3ecqP8sB5ZLegT4K+AxM0taIpPlvvY48MfxHyR1AAcAz1Yhtxz+\nHVgFfDtF3Z8SBgQzCf19WRYGeH4Hxia/j3leB8/tMWqd20c9r4PndsZHXoca5/Ya5/WKqcUTcPYB\nvgl83cwMuB14X269oKQFktqAO4E/V/SEiqixsmAasDk6AJ4GHJiR3HJJ8hvgxGjqr4EwGq9meh/C\nVO1nzOyRvN+n8fKNm0uq1JHPHwPXmtmBZjbfzPYHngFOIXv/ClGJz1cDXwUeMLNtGdnxHHCkpJbo\nStIbM5I73mxIau+GBN2rgYP18pOoLsgXWC6SFko6LPbTIuAJYB+FG1eR1CwpfmX7guj3kwnT79ur\nMOGXQKukSyKZjcAVhCvHtwOXK7rhO5bPuoH2KnQOI7pydiNhOUGO3xCtoycMAu6J6u4AHgC+Avws\ni5NQz+9DjEV+r0VeB8/tOWqd28dKf73n9prndahtbh8Heb1ixmomZ0o0rdVMuPpwLfDlqOxqwlTu\nKkkiTEWeZ2a3KdwsuFLSbsLU9CcqNSDqhH2EdYtLoysBKwnreGtBQb+jsgeArwOHAncBPy4kIC1m\ntp6Q4PP5AvAdSZ8kXIXLkouAf8377WbgfWTsXyEq8dnMfiupi3RXSoqS629mtk7SjcCjhAPD76qV\nPU5tSGrvCwmJeZhuM9sl6f3AbZJ6CH2iWqYCX4sOuHuAtYTlDVcBX1VYz9xEuCL2WLRNr6TfEXLT\nu6tRbmYm6XzgG5L+H+EkIJe3BghP5XlYUj/wX4R94CpCDDZmuH77CuCvY98/AHxb0scIeebPY2U3\nEJbznFqFPs/vIxn1/F6jvA6e22ua22ugv65z+zjK6zC2ub3meT0LFAZlEx+Fp6n8l5lV+wQlpwoU\nlmt81MzOrrUt+UiaQ5gOP9zMBquUVfP+Nh5sKIakqWa2I0qSVwJPmdm/jaH+5YS+uHKsdDqjw3jv\n6/WA5/axodb60+C53RkvjIcHD4w6kv6ScBPaJ2ttizM+iaai7wP+IYODYM3723iwIQV/EV0peoyw\n1OQ/a2yP8wrkFdLXnRoxkXJ7rfWXged2Z1xQNzM5juM4juM4juPUB3Uxk+M4juM4juM4Tv3ggxzH\ncRzHcRzHcSYUPshxHMdxHMdxHGdC4YMcx3Ecx3Ecx3EmFBN2kCNpf0l3SXpc0mOS/ib6faakOyU9\nFf2fEf1+uKQVkvokfTRP1nRJP5T0pKQnci+fchzHccaWrHK7wgsGH4z9dUn6UK38chzHcbJlwj5d\nTdJ+wH5mtkpSO/BbwsvYlgAvmdnnJX0cmGFmfy9pFuHt2OcB28zsSzFZ3wHuMbOrJU0CWs2sc6x9\nchzHqXeyzO0xmY3ABuAkM3turHxxHMdxRo8JO5NjZs+b2aroczfwBDAXOBf4TlTtO0RvoTazzWb2\nANAflxO9Sff1wH9H9Xb7AMdxHKc2ZJXb83gj8Hsf4DiO40wcJuwgJ46k+cCrCS8Em21mz0dFm4DZ\nJTY/CNgCfFvS7yRdLalttGx1HMdx0lFlbo9zIeEli47jOM4EYcIPciRNBW4GPmRmXfEyC2v1Sq3X\nawKOA/7DzF4N9AAfHw1bHcdxnHRkkNtzciYBbwNuytxIx3Ecp2ZM6EGOpGbCQfD7Zvaj6OcXojXd\nubXdm0uIWQ+sN7P7ou8/JAx6HMdxnBqQUW7P8RZglZm9kL2ljuM4Tq2YsIMcSSLcR/OEmX05VvRT\n4NLo86XALcXkmNkmYJ2khdFPbwQez9hcx3EcJwVZ5fYYF+FL1RzHcSYcE/npaicD9wCPAIPRz58g\nrN2+ETgAeA54p5m9JGlfYCXQEdXfARxpZl2SFgFXA5OAp4E/N7NtY+mP4ziOk3lubwP+ABxsZtvH\n1hPHcRxnNJmwgxzHcRzHcRzHceqTCbtczXEcx3Ecx3Gc+sQHOY7jOI7jOI7jTCh8kOM4juM4juM4\nzoTCBzmO4ziO4ziO40wofJDjOI7jOI7jOM6Ewgc5juM4juM4juNMKHyQ4ziO4ziO4zjOhOL/A5Se\n8bw0ZUUbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe301c861d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "import matplotlib.dates as mdates\n",
    "import os\n",
    "%matplotlib inline\n",
    "\n",
    "fn = join(\"data_IJGI18\",\"observations.csv\")\n",
    "# from db (to file)\n",
    "if False:\n",
    "    import psycopg2\n",
    "    with open(os.path.join(os.environ[\"HOME\"],\".pgpass\"),'r') as f:\n",
    "        pgpass = f.readline().replace(\"\\n\",\"\")\n",
    "    host, port, db, user, password = pgpass.split(':')\n",
    "    conn = psycopg2.connect('postgres://{}:{}@{}/{}'.format(user,password,host,db))\n",
    "\n",
    "    data = pd.read_sql(\"select distinct date, sat from bavaria order by date\",conn)\n",
    "    \n",
    "    # add categorical id for easier plotting\n",
    "    data[\"sat_id\"] = data[\"sat\"].astype('category').cat.codes\n",
    "    data.to_csv(fn)\n",
    "    # from file\n",
    "else:\n",
    "    data = pd.read_csv(fn,index_col=0)\n",
    "    \n",
    "dates=data[\"date\"].sort_values()\n",
    "sat=data[\"sat\"].astype(\"category\")\n",
    "#color=scl.sort_index()[\"cloud\"]\n",
    "\n",
    "if type(dates.values[0])==str: #parse to datetime\n",
    "    X = [datetime.strptime(d, '%Y-%m-%d') for d in dates.values]\n",
    "else: # assume already datetime\n",
    "    X = dates.values\n",
    "Y = data[\"sat_id\"]\n",
    "fig, ax = plt.subplots(figsize=(14,1))\n",
    "sc = ax.scatter(X, Y,\n",
    "           marker='o', s=80, edgecolors=\"#000000\", linewidth=.5, alpha=.5, cmap=\"Blues_r\", vmin=0, vmax=1)\n",
    "\n",
    "#cbar = plt.colorbar(sc,aspect=3, ticks=[0, .5, 1])\n",
    "#cbar.ax.set_yticklabels(['0% clouds', '50% clouds', '100 % clouds']) \n",
    "\n",
    "xticks = [datetime(year,month,1) for month in range(1,12,2) for year in [2016,2017]]\n",
    "ax.xaxis.set_ticks(xticks)\n",
    "ax.format_xdata = mdates.DateFormatter('%Y-%m-%d')\n",
    "# everything after this is turning off stuff that's plotted by default\n",
    "\n",
    "years = mdates.YearLocator()   # every year\n",
    "months = mdates.MonthLocator()  # every month\n",
    "monthsFmt = mdates.DateFormatter('%b')\n",
    "yearsFmt = mdates.DateFormatter('%Y')\n",
    "\n",
    "# format the ticks\n",
    "ax.xaxis.set_minor_locator(years)\n",
    "ax.xaxis.set_minor_formatter(yearsFmt)\n",
    "ax.xaxis.set_major_locator(months)\n",
    "ax.xaxis.set_major_formatter(monthsFmt)\n",
    "ax.xaxis.set_tick_params(which='minor', pad=20)\n",
    "\n",
    "#ax.yaxis.set_visible(False)\n",
    "ax.spines['right'].set_visible(False)\n",
    "ax.spines['left'].set_visible(False)\n",
    "ax.spines['top'].set_visible(False)\n",
    "\n",
    "#ax.xaxis.set_ticks_position('bottom')\n",
    "\n",
    "ax.xaxis.grid(True, which=\"major\")\n",
    "#ax.yaxis.grid()\n",
    "\n",
    "plt.yticks(sat.cat.codes.unique(), sat.cat.categories)\n",
    "\n",
    "plt.xlim(datetime(2015,12,1), datetime(2017,12,31))\n",
    "\n",
    "ax.set_title(\"{} Sentinel 2 A and B observations over 2016 and 2017\".format(len(dates)))\n",
    "\n",
    "plt.ylim(-.5, 1.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Per Class accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   precision    recall  f1-score   support\n",
      "\n",
      "       sugar beet       0.91      0.78      0.84    153010\n",
      "       summer oat       0.73      0.65      0.69     73695\n",
      "           meadow       0.89      0.85      0.87    475047\n",
      "             rape       0.96      0.93      0.94    238215\n",
      "              hop       0.95      0.81      0.87    103868\n",
      "     winter spelt       0.60      0.65      0.63     68519\n",
      " winter triticale       0.66      0.56      0.61    128810\n",
      "            beans       0.93      0.75      0.83     55432\n",
      "             peas       0.81      0.59      0.68     15621\n",
      "          potatoe       0.93      0.81      0.86    265914\n",
      "         soybeans       0.84      0.80      0.82     46498\n",
      "        asparagus       0.85      0.78      0.82     39633\n",
      "     winter wheat       0.89      0.94      0.91   1589217\n",
      "    winter barley       0.94      0.90      0.92    512710\n",
      "       winter rye       0.81      0.53      0.64     72722\n",
      "    summer barley       0.82      0.86      0.84    163711\n",
      "            maize       0.92      0.97      0.94   1795452\n",
      "\n",
      "      avg / total       0.90      0.90      0.89   5798074\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data16 =np.load(join(quantevalpath,\"2016\",\"truepred.npy\"))\n",
    "data17 =np.load(join(quantevalpath,\"2017\",\"truepred.npy\"))\n",
    "data = np.vstack((data16,data17))\n",
    "\n",
    "y_true = data[:,0]\n",
    "y_pred = data[:,1]\n",
    "\n",
    "print(skmetrics.classification_report(y_true, y_pred, labels=ids, target_names=names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def calc_avg(y_true,y_pred,function):\n",
    "    micro = function(y_true, y_pred, labels=ids,average='micro')\n",
    "    macro = function(y_true, y_pred, labels=ids,average='macro')\n",
    "    weighted = function(y_true, y_pred, labels=ids,average='weighted')\n",
    "    return micro,macro,weighted\n",
    "\n",
    "dec = 2\n",
    "prec,rec,fscore,sup = skmetrics.precision_recall_fscore_support(y_true, y_pred, beta=1.0, labels=ids)\n",
    "\n",
    "avg_prec = skmetrics.precision_score(y_true, y_pred, labels=ids,average='weighted')\n",
    "avg_rec = skmetrics.recall_score(y_true, y_pred, labels=ids,average='weighted')\n",
    "avg_fscore = skmetrics.f1_score(y_true, y_pred, labels=ids,average='weighted')\n",
    "\n",
    "if write_flag:\n",
    "\n",
    "    metrics = np.column_stack((classes,np.around(prec,dec),np.around(rec,dec),np.around(fscore,dec),sup))\n",
    "    with open(join(quantevalpath,\"metrics.csv\"), 'w') as f:\n",
    "        f.write(b'id,name,precision,recall,fscore,support\\n')\n",
    "        np.savetxt(f, metrics, fmt='%s', delimiter=', ', newline='\\n')\n",
    "        f.write(b',,,,,\\n'.format(prec=avg_prec, rec=avg_rec, fsc=avg_fscore))\n",
    "        f.write(b',weight. avg,{prec},{rec},{fsc},\\n'.format(prec=np.around(avg_prec,dec),rec=np.around(avg_rec,dec),fsc=np.around(avg_fscore,dec)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cm = skmetrics.confusion_matrix(y_true, y_pred)\n",
    "\n",
    "#sklearn.metrics.precision_recall_fscore_support(y_true, y_pred, beta=1.0, labels=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAADxCAYAAAAqYhcBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAHUVJREFUeJzt3X20ZFV55/Hvr283IAgoL2lpaMBo\nR0MY04wd1BnHwLikmw4TnLUcB8aVwQna6tI1yzV5U5NRR5MJszKJSxdOoNVOayKgk4j2chgaNGbQ\nJAoN4VVFEMF+QRq6Qd7sl3vrmT/OuU317aqzz6176tQ5Vb8P66yqOmfXObsvdffdtc+zn62IwMzM\nxtuiUVfAzMyGz429mdkEcGNvZjYB3NibmU0AN/ZmZhPAjb2Z2QRwY29m1lKSNkjaKenuVFk39mZm\n7bURWFOmoBt7M7OWioibgN1lyi4ecl3MzKzL6nOPil27Z5Llbr1z7z3Anq5d6yNi/aDXdWNvZlaj\nXbtnuHnzqclyUyfdtyciVlV1XTf2ZmY1CqBDp/brurE3M6tREOyP9DBO1XyD1sysZp0S/5Uh6Wrg\nH4GXSdom6dJ+Zd2zNzOrURDMVJRaPiIuLlvWjb2ZWc061L+OiBt7M7MaBTDjxt7MbPy5Z29mNuYC\n2D+C5WDd2JuZ1SgID+OYmY29gJn623o39mZmdcpm0NbPjb2ZWa3EDKr9qm7szcxqlN2gdWNvZjbW\nsjh7N/ZmZmOv4569mdl4G1XP3lkvJ4SkD0v6q/z56ZJCkv/Ym9UsEDMsSm5V8y+7mVnNPIxjSFoc\nEdOjroeZDUcg9sVU7df1ME4DSHpQ0u9JuhN4RtKpkv5G0qOSfiTpP3eVnZL0AUk/lPSUpFslLc+P\nfVzSVklP5vv/1cj+UWbWUzapalFyq5ob++a4GPg14DjgWuAO4GTg9cB7Ja3Oy/2XvOxa4BjgN4Fn\n82O3ACvzc1wF/G9JR9T1DzCzcmbyiVVFW9Xc2DfHJyJiK3AmcGJEfCQi9kXEA8CngIvycm8D/iAi\n7o3MHRGxCyAi/ioidkXEdET8KXA48LJR/GPMrLcIMROLklvVPGbfHFvzx9OAZZKe6Do2BXwzf74c\n+GGvE0j6beBSYBnZt8VjgBOGUlszG1jHk6om2mwevK3AjyJiRZ9yW4GXAHd378zH53+XbNjnnojo\nSHocRvCpMrO+shu09Te9HsZpnpuBp/Ibts/Lb8ieKelX8uOfBj4qaYUyr5B0PHA0MA08CiyW9EGy\nnr2ZNYhv0BoAETEDXEB2o/VHwGNkDfyxeZE/A74I3AA8CXwGeB6wGbge+AHwELCH54aGzKxBZkLJ\nrWqKESyPZWY2qV7yz46Ky7788mS5N7/0tlsjYlVV1/WYvZlZzTpDiLZJcWNvZlajLBGaG3szs7EW\niP0jSJfgxt7MrEYRDGXSVEojG/sTjpuK05cvKSzzgzuPrKk2E04logKadpM/UeU98Qz7Yq/nHwyZ\npLcAl0TEeYlyVwDbI+Kj9dRs1NS+SVWS1gAfJ5vh+emIuGzO8cOBzwGvBHYB/z4iHkyd9/TlS7h5\n8/LCMquXrRyw1nbAovRXSU2ly8T0/vS1avyDoMXFH+tvT2+uqSaTLSI+D3y+RLl31lCdxghG07Mf\n+IqSpoBPAucDZwAXSzpjTrFLgccj4qXAx4D/Mej1zKx+XuBmOEaxeMlCzng2cH9EPBAR+4BrgAvn\nlLkQ+Gz+/K+B10tlxgXMbJjytNrvl/RdSY9L+gtJR0g6R9K2fAb3T4C/yMtfIOl2SU9I+gdJr+g6\n13JJX8pTcu+SdHm+/62SvpU/l6SPSdqZp+C+S9KZ+bGNkv6w63xvl3S/pN2SNkla1nUsJL1T0n15\nXT7ZtjYlEJ1Ib1VbSGN/MgfP0NyW7+tZJl+Q46fA8b1OJmmdpC2Stjy6a2YB1TKzkt4CrCbLtfQL\nwB/k+19Elib7NGCdpLOADcA7yH5/rwQ2STo8/4b/VbJZ26eT/c5f0+Na5wGvy69zLPBmsqHdg0j6\n18Af58dPys8793wXAL8CvCIvt5oWCWB/LE5uVWtMuoSIWB8RqyJi1YnH1x+WZDaBLo+IrRGxG/gj\nsnUSADrAhyJib0T8DFgHXBkR34mImYj4LLAXeDXZN/xlwO9ExDMRsScivtXjWvvJ8je9nGzm/vci\n4uEe5d4CbIiI2yJiL/B+4DWSTu8qc1lEPBERPwa+QZZapEXSueybls9+O1m63Vmn5Pt6lsnH/o6l\nx19zMxuJ7m/mD5E12gCPRsSermOnAb+VD5s8kaffXp6XXw48lFpKMyL+Fric7D7fTknrJfVK1Lcs\nr8vs+54mazO6Rw1+0vX8WeD5RddumiCbQZvaqraQM94CrJD0YkmHkS2usWlOmU3AJfnzNwF/G07G\nY9YU3Z21U4Ed+fO5v6NbgT+KiBd0bUdGxNX5sVPL3MiNiE9ExCvJAjp+AfidHsV2kP1xAUDSUWRD\nR3M7kq02ip79wANDETEt6T1k2RanyL563SPpI8CWiNhElpHxLyXdD+zmudWWCv3gziOToZWbd9ye\nPM/qk89KX6xhf3sWHXVUutBM+p5GZ8+eZBk66fNEiTKlYvErsvikFyXLTD/8k+ICzfpfPkrvlvRV\nst7x7wNf6FPuU8C1kr5GloL7SOAc4Kb89cPAZZI+BMwAr4yIv+8+QZ6iexFwG/AMWVbWTo9rXQ1c\nLekq4HvAfwe+UyZkuy0i1L7cOBFxHXDdnH0f7Hq+B/h3C7mGmQ3NVWSpspcBXwH+kGwM/iARsUXS\n28mGYVYAPwO+BdwUETOS/g3wCeDHZH9KrwL+fs5pjiELv/55soZ+M/AnPa71NUn/Ffgb4IXAP1Cy\nk9gW2Q1ap0sws/rcEhF/PGff35HdfztIRFxPtl7CIfIbpW/ssX8jsDF//nWy6Jle73/rnNdXAFf0\nKas5r9/aq1yzyekSzMzGXXaDtmXpEszMbP6c4tjMahERp4+6DpNqdgZt3dzYm5nVbBgLiqe4sTer\nSVHqbqfsTigK7x1F+HRBdZ6Kxx+LiBP7HY+A/R039matkkrz3a0odbdTdlOYcrso1XZhiu0h/SEo\nSqN94/5rHup7kNlhHDf2GQktOaywyNozfjV5ms3b/1+yTKmJVyUset7z0oVKTD7q/Cw9GWrxSUuT\nZeKRnen6lKDDD09fa186n32UmAhG9Jpjc7DO088ky6Ty2VM4sb+8rjTfbyBLBHiLpE0R8d1qrmDj\nahgzZFMakwjNrIXKpPk2O8hs6GXdKY6b2bM3a4deab5f1V1A0jqyrJGcerJ/3QwY0TCOe/ZmQ+TU\n3dZLJ1+HtmirmrsaZoMrk+bb7CBZNI5z45i1yYE032SN/EXAf+hXuCiba1EW12QQQc2hh4WZWRM3\n4guzsRZkWC3MvjqkrKtFGVaTmVULeFKVWcv0S/M94mpZCwxjmCZl4MZe0nLgc8BSshvM6yPi43PK\nnEOWOvVH+a4vRcRHBr2mWdP0SvNtVqSNidCmgd+KiNskHQ3cKunGHjHG34yICxZwHTOzsdKqSVX5\nYsEP58+fkvQ9slC0hU8oiSD27yssMvNE8XEoNytx845/quQ8nWefTZapyvT2HelCFYnpEjOQyoyZ\nVjSu3HnqqXShVH28UpWNUISYbmvoZb7y+1nAd3ocfo2kOyT9X0m/VHCOdZK2SNqyn71VVMvMrJFa\nOalK0vPJlhB7b0Q8OefwbcBpEfG0pLXAl8mWNTtERKwH1gMco+Pc9zKzsdTGMXskLSFr6D8fEV+a\ne7y78Y+I6yT9L0knRMRjC7muWSsV5HwqyvWUyvE0aH6nwnxOBUNhRfmbUnmbBs3ZVJSjqSg3U2FO\npkQupqI8TIX5l9KpotrV2EsS8BngexHxZ33KvAh4JCJC0tlkw0a7Br2mmVnbtTHO/l8CvwHcJWl2\nRsgHgFPhwKLBbwLeJWmabEX6iyJGkXzazKw5WhVnHxHfojCFP0TE5cDlg17DzGzcRMC0Fy8xMxt/\nbRvGMTOzeWrjmP1olZnIo/RXpXITr/onqZrPeSqbfFTjJKbUimFAcgJcdqL2/dvNhiXc2JuZjb9W\n3aA1s3kqSANSlP4j9a2xKOVH0XuHkeJjWKk8CtN2FH3bW8C3vMLUHAtIqxzhMXszswkgZhyNY2Y2\n/jxmb2Y25lqZG8fMzOYpRhMw5sbezKxmjsYxMxtz4Ru086OpqWSZUqsslVDVxKu1L39dsszMk3OX\nBOihxu+ApSZMLUr/v6BTkGo2V5g2drY+RSlr26wolC8xObDo81n0uSz8XA8azphcJWywz27R5L7C\nz+hCwjKHFNJZwdsH0trG3sysrVoZjSPpQeApYAaYjohVc44L+DiwFngWeGtE3LbQ65qZtVFESxv7\n3LkFq0+dT7YU4QrgVcCf549mZhNpFKGXddwluBD4XGS+DbxA0kk1XNfMrJEi0lvVqmjsA7hB0q2S\n1vU4fjKwtev1tnzfQSStk7RF0pb97K2gWmZmzROITmdRcqtaFcM4r42I7ZJ+DrhR0vcj4qb5niQi\n1gPrAY7Rcc5Ra2ZjaxQN3IIb+4jYnj/ulHQtcDbQ3dhvB5Z3vT4l32fWeqkAhdLnKQglXkgI8aBh\nmUVhwoXhwUOKKSwMrywK/S0I+U2F+g4tzHdEN2gX9F1B0lGSjp59DpwH3D2n2CbgPyrzauCnEfHw\nQq5r1jDnRsTKQRt6m0BRYqvYQnv2S4Frs+hKFgNXRcT1kt4JEBFXANeRhV3eTxZ6+Z8WeE0AotOs\nVY3WXPgbyTLXf/8vk2VqXfGqIlpS4mM0k66zXv7SZJm4594yVTJrtNaFXkbEA8Av99h/RdfzAN69\nkOuYNdhsgEIAV+b3nsz6CqDTaVljb2bFAQp5hNo6gCM4clR1tCYJoG1j9maTrjtAAZgNUOg+vj4i\nVkXEqiUcPooqWgO1Nc7ebCKVDFAwO1QLb9CaTbKeAQqDnKgw4GBImSSLggqKggkGzpYJQ+myFgYI\nFAQGpAIChhcMoPbdoDWbZP0CFMySnOLYzGzMBYSjcczMJoEb++ckxv6mjj0meYqZxx+vqjZJcctd\nyTJVrXi1+pRXlqhRJ12kovHT2FtN4rq4+/vpQmVWxYoxXc3KxoeHcczMJoAbezOzMTeiSVVu7M3q\n1Gd4smhYcljDkUVDjwMvYp4cYiwYXhxwWHHQYcTksGHRkOEChwq94LiZ2SRwNI6Z2fjTCHr2A6dL\nkPQySbd3bU9Keu+cMudI+mlXmQ8uvMpmZi1WJlVCk9IlRMS9wEoASVNkq09d26PoNyPigkGvY2Y2\nXtTqG7SvB34YEQ9VdD4zs/HV4hu0FwFX9zn2Gkl3ADuA346Ie5JnU/GanFDdRJ6mKTNhavO2W9Pn\nKbPiVRtFicliZk03go/xght7SYcBvw68v8fh24DTIuJpSWuBLwMr+pzHizzYeCvoxLSp81LUIUl1\nRFrVCRlWx6LFi5ecD9wWEY/MPRART0bE0/nz64Alkk7odZKDFnmQF3kws/GlSG9Vq6Kxv5g+QziS\nXqQ82beks/Pr7argmmZm7dWmaBw4sDrPG4B3dO17JxxYdPxNwLskTQM/Ay7KFyA3M7MaLaixj4hn\ngOPn7Lui6/nlwOULuYaZ2bgZxaQqz6A1M6tT4HQJZmYTwT17M7Px52GcnLQIHV4cftl59tnkeRYd\ncUSyTGfPntL1WrDE6lsAdNKpU9f+0rnJMpt3fCNZ5vzzL07X594fJYt0ysSIV3VfXukAMk0lfs7T\n1VRlvoo+10Wf59TneCif4aLPasFnNPXZLPpcFn4eCz6HhZ+/hXzuCj5rhZ+x/SXO7cbezGwCuLE3\nMxtvw5o0leLG3sysbo7GMTMbf+7Zm5lNAjf2ZmZjzmP2Zs0kaQNwAbAzIs7M9x0HfAE4HXgQeHNE\nPF50nuh06DzzzLyvXyq0tWoDhizOPF74I2D1yWf1PbZ5e78lMRLrPAwr3VZBiOmCsx+3aQ1aswmy\nEVgzZ9/7gK9HxArg6/lrs1LUSW9Va2TPftAe0Fwj6REVqagHkuo9QXEPalZRT+rAeUqsnDW0nlUv\nJSadJXtd86xuRNwk6fQ5uy8Ezsmffxb4O+D35ndms/o0srE3a4GlEfFw/vwnwNJehbwCm/XU1GEc\nSRsk7ZR0d9e+4yTdKOm+/PGFfd57SV7mPkmXVFVxs6bI12jo+et70ApseAU248AN2qauVLWRAcYs\n85tYHwJeBZwNfKjfHwWzlnlE0kkA+ePOEdfH2mQEK1WVauwj4iZg95zdF5KNVZI/vrHHW1cDN0bE\n7jxS4UYO/aNh1kabgNlvqpcAXxlhXaxtWrYsYZkxy5OBrV2vt+X7DuGxTWsqSVeT3Yw9QdI2sm+r\nlwFflHQp8BDw5qFVIHUDvChDZd2rgC6aKj5ecPd89bKVfY9t3nHrQO9rIjGcaJuUSm7QRkRICxtl\nioj1wHqAY3Sc16m1xoiIfrl3X19rRWw8jGhS1ULi7MuMWW4Hlne9PiXfZ2Y2uZo6Zt9HmTHLzcB5\nkl6Y35g9L99nZja5mjpmP58xS0mrgHdGxNsiYrekjwK35Kf6SETMvdE7PGXGK8usHlX3uGdKalwU\nSs3nLjPWWTRWOp/zmNlzGpsbZz5jlhGxBXhb1+sNwIaBamdmNo6a2tibmVlFosXROGa2MFMnHN/3\nWOeJnxa+NzoF3cTon0vo0Xe9pu+xF967r++xxd+4ragy/Y+lFAypFodl3t732JrTzu57bNExzy+s\nTuzrv3J47Ov/86HM+u/u2ZuZjb/GjtmbmVmF3NibmY25IYVWprixNzOrkfAwjpnZRHBjPw9F0Quz\nUlEMkIhkOFAovTpSUWTDrKIIh1mFkQ4H6lNR3FaJCWXlJl71j4aYVRQVMSsVHQHFERLPlUn8nPeW\nmEhnNkxu7M0m08xju/ofLDPLewAnXnlz32OLT+2ZnBaA6YbNKC/qSFz/UP9/45oXv6rwvLF/uuDg\nAjtcbuzNzMbciLJeurE3M6ubG3szs/HndAlmZhOgkYuXSNogaaeku7v2/Ymk70u6U9K1kl7Q570P\nSrpL0u2StlRZcTOzViqTy35Ei5ds5NBFwm8EzoyIVwA/AN5f8P5zI2JlRKwarIpmZmOmiYuXRMRN\nkk6fs++GrpffBt5UbbXM7IAhhTpOveS0vsc6hx82lGsOQ8z0nwdTnC3zO4XnHdaiPG2eQfubwBf6\nHAvghnwx8ivzRcV7krQOWAdwBEcmL1oYl/zcSdNlKlIUszyrKHZ5VtNimMsoM2GqKN75wHkScc+Q\niH0+UChx96uFP2MbLyozmbNiC2rsJf0+MA18vk+R10bEdkk/B9wo6fsRcVOvgvkfgvUAx+g4/zaa\n2XgaUSK0gRccl/RW4ALgLRG9u0oRsT1/3AlcC6S7gGZmY06R3qo2UGMvaQ3wu8CvR8SzfcocJeno\n2efAecDdvcqamU2UJkbjSLoa+EfgZZK2SboUuBw4mmxo5nZJV+Rll0m6Ln/rUuBbku4Abgb+T0Rc\nX/0/wcysXUbRsy8TjXNxj92f6VN2B7A2f/4A8MsLqp2Z2ThqaTSOmZmVFU6XYNZIkjaQBSPsjIgz\n830fBt4OPJoX+0BEXNf7DGlTv7ii77GZ79036GkLzdz3wGBvLAhp1uIlhW8tiomnk143ovdJB2s5\nU3H0Res0LCQGf1Rx9gNH45hNkI0cOosc4GP57PCVC2nobQJFpLeKjXfPvsbJM0WzEWe1aVbifBT2\n1nLlVrwqntFY9jxV6zWL3Gwh3LM3a5f35MkAN0h6Ya8CktZJ2iJpy3721l0/a6IGJ0Izs0P9OfAS\nYCXwMPCnvQpFxPqIWBURq5ZweJ31swZTJ71VbbyHccyGJCIemX0u6VPAV0dYHWuZUUTjuGdvNgBJ\nJ3W9/Ld4driVFfgGrVkT5bPIzwFOkLQN+BBwjqSVZL+6DwLvKHEitKT3TfrOAz/u+7ZFRxxReNrO\n3v73AhYd2T+DbOeZZ/qfdEgZY7WkoMnpTPU9tOjYo/u/7amn+x6Lffv6Xy/RoK4++ay+xzbv+Ke+\nx6ZO6nvogLamODYba/OZRW5Wiht7M7Px1ubFS8zMrKyI9i1eMkpF08tnDWuaec9rDTr1fK4SY6Wp\nKelQbqLTwNPTD7lYNaEF5SZe9Z/CPp/zmI2Ue/ZmZuOvkTNo89mBOyXd3bXvw5K257nsb5e0ts97\n10i6V9L9kt5XZcXNzFopgE6kt4qV6dlvJFus5HNz9n8sIv5nvzdJmgI+CbwB2AbcImlTRHx3wLqa\ntdpTsfuxG/dd9VDXrhOAx4Z60f5Rib08V59B25qCSMd5Ovhn82j/gkNz8M/goPokwivTibKaOIyz\ngCRQZwP354uYIOka4ELAjb1NpIg4sfu1pC0RsWpU9ZmrSfVpUl2g+vo0chinQCoJ1MnA1q7X2/J9\nPTlhlJlNCnUiuVVt0Ma+VBKo+XDCKDObCCPKejlQNE7JJFDbgeVdr0/J95lZZv2oKzBHk+rTpLpA\nhfXJJlXVP44zUM++ZBKoW4AVkl4s6TDgImDTINczG0cR0agGrUn1aVJdYAj16ZTYKpbs2c8nCZSk\nZcCnI2JtRExLeg+wGZgCNkTEPaVqVZAwalZR4qhZqQRSUJxE6sB5CpJJHThPUVKpWUNKLtXzUkUJ\np2YVJJ6aVZSA6sBpChJRzSpMSHWgULq3U5ScalZRkiqAs1c/m66L2RCNomdfJhqndBKoiNgBrO16\nfR3gtTnNzGYNaUw+xfnszWrWtMmGkh6UdFc+QXLLCK7fa+LmcZJulHRf/thz2cca61NqImk56Uic\nJkXjmNkAuiYbng+cAVws6YzR1gqAcyNi5Yhi2zcCa+bsex/w9YhYAXw9fz3K+kA2kXRlvi1sxGIE\ni5e4sTer14HJhhGxD5idbDixIuImYPec3RcCn82ffxZ444jrU+EFRrMGrRt7s3rNa7JhTQK4QdKt\nktaNuC6zlkbEw/nznwBLR1mZXGoiaXnu2ZvZCLw2Iv452dDSuyW9btQV6hYRI7qleZBqJ5KOYFKV\nG3uzejVusmFEbM8fdwLXkg01jdojs/N58sedo6xMRDwSETMR0QE+xQJ/Rup0klvV3Nib1atRkw0l\nHSXp6NnnwHn0niRZt03AJfnzS4CvjLAuZSeSlhM0c1LVKIwkFWyR+aWJnXVonav6alZdGtm5Dq3z\nKFLLFilIOzsrkX4WyqSgHZIFTTYcjqXAtcom/C0GroqI6+usQJ+Jm5cBX5R0KfAQ8OYR16fnRNKB\nzk+MZFKVYgQXna+mpTstw3UevrbV1wzg2KOWxat/MX0f/IZb/9utVX6+G9mzNzMba01Ml2BmZhWa\nHbOvWVsa+0ZlwCvJdR6+ttXXDGAo0TYprWjsm5butAzXefjaVl+zzHAmTaW0orE3MxsbgRt7M7OJ\nMIIx+8ZPqmpaOtiUUaeLLaNpKWXLGH7aWbP6KCK5Va3RjX2D08GmjDJdbBkbaVZK2TI2Muy0s2Z1\ncSK0Qzgd7BA0LaVsGUNPO2tWlwiY6aS3ijW9sW9iOtiUJqaLLaOJKWXLqC7trFld3LMfC41OF1tG\nQ1LKllFt2lmzurixP0Tj0sGmNDRdbBmNSilbRtVpZ81qEUAn0lvFmt7YNyodbEqD08WW0aiUsmVU\nmnbWrDYB0UlvFWt0nH0D08GmjDxdbBlNSylbxrDTzprVJhjKDdiUVqQ4NjMbF8cetjT+xdKLkuWu\n3/YJpzg2M2s1p0swMxt3ToRmZjb+AnCKYzOzCeCevZnZuIuRROO4sTczq1NADCGOPsWNvZlZ3YYw\nQzbFjb2ZWd08Zm9mNuYiHI1jZjYR3LM3Mxt3QczM1H5VN/ZmZnWaTXFcMzf2ZmZ1G0HoZdPz2ZuZ\njZUAohPJrQxJayTdK+l+Se8rKuvG3sysTlHN4iWSpoBPki2BegZwsaQz+pX3MI6ZWc0qukF7NnB/\nRDwAIOka4ELgu70Ku7E3M6vRUzy++Wvx1yeUKHqEpC1dr9dHxPqu1ycDW7tebwNe1e9kbuzNzGoU\nEWtGcV2P2ZuZtdN2YHnX61PyfT25sTcza6dbgBWSXizpMOAiYFO/wh7GMTNroYiYlvQeYDMwBWyI\niHv6lVeMIEeDmZnVy8M4ZmYTwI29mdkEcGNvZjYB3NibmU0AN/ZmZhPAjb2Z2QRwY29mNgH+PyLm\n640IoZssAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe3401659d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "fig,axs = plt.subplots(1,2)\n",
    "axs[0].imshow(cm/cm.max(axis=1).astype(float))\n",
    "axs[0].set_title(\"recall\")\n",
    "\n",
    "cax = axs[1].imshow(cm/cm.max(axis=0).astype(float))\n",
    "axs[1].set_title(\"precision\")\n",
    "cbar = fig.colorbar(cax, ticks=[0, 1], orientation='vertical')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "classnames = classes\n",
    "\n",
    "import csv\n",
    "\n",
    "def write_flat_cm(confusion_matrix, outfile):\n",
    "    precision = confusion_matrix/(confusion_matrix.sum(axis=0)+1e-10)\n",
    "    recall = confusion_matrix/(confusion_matrix.sum(axis=1)+1e-10)\n",
    "\n",
    "    outcsv = \"\"\n",
    "    rows,columns = confusion_matrix.shape\n",
    "    for c in range(columns):\n",
    "        for r in range(rows):\n",
    "            row=\"{r} {c} {absolute} {precision} {recall}\".format(r=r+1,c=c+1,absolute=int(confusion_matrix[r,c]), precision=precision[r,c], recall=recall[r,c])\n",
    "            outcsv+=row+\"\\n\"\n",
    "\n",
    "    with open(outfile,\"w\") as f:\n",
    "        f.write(outcsv)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if write_flag:\n",
    "\n",
    "    # Absolut\n",
    "    with open(join(quantevalpath,'confusion_matrix_abs.csv'), 'wb') as f:\n",
    "        f.write(\", \".join(classnames)+\"\\n\")\n",
    "        np.savetxt(f, cm, fmt='%d', delimiter=', ', newline='\\n')\n",
    "\n",
    "    # Precision\n",
    "    with open(join(quantevalpath,'confusion_matrix_prec.csv'), 'wb') as f:\n",
    "        f.write(\", \".join(classnames)+\"\\n\")\n",
    "        np.savetxt(f, cm/cm.sum(axis=0).astype(float), fmt='%f', delimiter=', ', newline='\\n')\n",
    "\n",
    "    # Recall\n",
    "    with open(join(quantevalpath,'confusion_matrix_rec.csv'), 'wb') as f:\n",
    "        f.write(\", \".join(classnames)+\"\\n\")\n",
    "        np.savetxt(f, cm/cm.sum(axis=1).astype(float), fmt='%f', delimiter=', ', newline='\\n')\n",
    "\n",
    "    # write these out as flat confusion matrix for tikz pgf plotting...\n",
    "    write_flat_cm(confusion_matrix=cm, outfile=join(quantevalpath,'confusion_matrix_tikz.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write flat confusion matrix for paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import csv\n",
    "\n",
    "filename= 'confusion_matrix.csv'\n",
    "\n",
    "def write_flat_cm(confusion_matrix, outfile):\n",
    "\n",
    "    with open(filename,'r') as f:\n",
    "        header = f.readline()\n",
    "\n",
    "    precision = confusion_matrix/(confusion_matrix.sum(axis=0)+1e-10)\n",
    "    recall = confusion_matrix/(confusion_matrix.sum(axis=1)+1e-10)\n",
    "\n",
    "    outcsv = \"\"\n",
    "    rows,columns = confmat.shape\n",
    "    for c in range(columns):\n",
    "        for r in range(rows):\n",
    "            row=\"{r} {c} {absolute} {precision} {recall}\".format(r=r+1,c=c+1,absolute=int(confmat[r,c]), precision=precision[r,c], recall=recall[r,c])\n",
    "            outcsv+=row+\"\\n\"\n",
    "\n",
    "    with open(outfile,\"w\") as f:\n",
    "        f.write(outcsv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize the Class activations of some tiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import ipywidgets as widgets\n",
    "from __future__ import print_function\n",
    "from ipywidgets import interact, interactive, fixed, interact_manual\n",
    "\n",
    "from PIL import Image\n",
    "from os.path import join\n",
    "import os\n",
    "\n",
    "#model=\"data/bavaria/models/fine_tuned/convlstm256_48px/eval/2017\"\n",
    "tiles = [int(t.split(\".\")[0]) for t in os.listdir(join(qualevalpath,\"2017\",\"ground_truth\"))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_class(ax, tileid, folder=\"ground_truth\", title=\"ground truth\",cmap=\"Accent\"):\n",
    "    #im = plt.imread(join(model,folder,str(tileid)+\".tif\"))[:,:,0]\n",
    "    im = Image.open(join(qualevalpath,\"2017\",folder,str(tileid)+\".tif\"))\n",
    "    arr=np.asarray(im)\n",
    "    ax.set_axis_off()\n",
    "    #arr[arr==0]=np.nan\n",
    "    #plt.set_clim(vmin=0, vmax=27)\n",
    "    #cmap=plt.get_cmap(\"gist_earth\")\n",
    "    #print cmap.get_clim()\n",
    "    if cmap=='tum': \n",
    "        colors = tumcolors\n",
    "        rs = np.zeros(arr.shape,dtype=int)\n",
    "        lut = [0]+ids\n",
    "        for i in ids:\n",
    "            np.place(rs, arr==i, lut.index(i))\n",
    "        arr=rs\n",
    "    else: colors = plt.get_cmap(cmap)(np.linspace(0,1,27))\n",
    "    colors[0]=np.array([1,1,1,1])\n",
    "    colored_im = colors[arr]\n",
    "    ax.imshow(colored_im)\n",
    "    ax.set_title(title)\n",
    "    #ax.colorbar()\n",
    "    \n",
    "def plot_activation(ax, tileid, folder=\"loss\", title=\"loss\",vmax=None):\n",
    "    \n",
    "    #ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "    ax.set_axis_off()\n",
    "    #im = plt.imread(join(model,folder,str(tileid)+\".tif\"))[:,:,0]\n",
    "    im = Image.open(join(qualevalpath,\"2017\",folder,str(tileid)+\".tif\"))\n",
    "    #np.asarray(im)\n",
    "    cmap = plt.get_cmap(\"inferno\")\n",
    "    image=cmap(np.asarray(im))\n",
    "    ax.imshow(image, cmap=\"inferno\",vmin=0, vmax=vmax)\n",
    "    ax.set_title(title)\n",
    "\n",
    "def plot_tile(tileidx):\n",
    "    tileid=tiles[tileidx]\n",
    "    \n",
    "    fig, axs = plt.subplots(5,4, figsize=(10,12))\n",
    "    \n",
    "    \n",
    "    plt.suptitle('Tile '+(str(tileid)))\n",
    "    \n",
    "    #axs = [ax for ax in ax_row for ax_row in axs]\n",
    "    axs_list=list()\n",
    "    for ax_row in axs:\n",
    "        for ax in ax_row:\n",
    "            axs_list.append(ax)\n",
    "\n",
    "    plot_class(axs_list.pop(0),tileid, folder=\"ground_truth\", title=\"ground truth\",cmap=\"tum\")\n",
    "    plot_class(axs_list.pop(0),tileid, folder=\"prediction\", title=\"prediction\",cmap=\"tum\")\n",
    "    plot_activation(axs_list.pop(0),tileid, folder=\"loss\", title=\"loss\",vmax=1)\n",
    "\n",
    "    flds = [\"1_sugar_beet\",\"2_summer_oat\",\"3_meadow\",\"4_rape\",\"5_hop\",\"6_winter_spelt\",\"7_winter_triticale\",\"8_beans\",\"9_peas\",\n",
    "    \"10_potatoe\",\"11_soybeans\",\"12_asparagus\",\"13_winter_wheat\",\"14_winter_barley\",\"15_winter_rye\",\"16_summer_barley\",\"17_maize\"]\n",
    "\n",
    "    names=[\" \".join(el.split(\"_\")[1:]) for el in flds]\n",
    "\n",
    "    #plot_activation(axs_list[0],tileid, folder=\"confidences/\"+flds[0], title=names[0])\n",
    "    #plot_activation(axs_list[1],tileid, folder=\"confidences/\"+flds[1], title=names[1])\n",
    "    #plot_activation(axs_list[2],tileid, folder=\"confidences/\"+flds[2], title=names[2])\n",
    "    for fld,name in zip(flds,names):\n",
    "        plot_activation(axs_list.pop(0),tileid, folder=\"confidences/\"+fld, title=name,vmax=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAEICAYAAAB/KknhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACN5JREFUeJzt3VuopXUZx/HfU2KQQqVJk5pZXkQU\nnaCDKCSlFwYiSUSkoh3Mmy666EBUmhUJ0lUXkURpaZllFhlMZBcV2WBUJgQVhDk2HmpKpTykUG8X\ne43t2XvNPqzZex328/nAwD6swzvKd//X+8z7X7uGYQjQy9NmfQDA9AkfGhI+NCR8aEj40JDwoSHh\ns66q+mRVXT/l5zy5qoaqOmKaz9uF8DksVXVGVe3bgse5u6rO3IpjYn3CX0CLtgou2vF2IPw5UVWv\nqao7qupfVfXtqrqxqj4z+t4ZVbWvqj5SVQ8kuWb09Uuq6k9V9WBVfb+qjh99fdXL5Kr6SVW9d/Tx\nxVX186r6XFU9VFV/rqqzl932RVX109Gx3JrkuYc45qOS7E5yfFU9Mvpz/OjU4Kaqur6q/pnk4qq6\n9sDfZ/nfafTxdUlOSnLL6DE+vOxpzq+qe6rq71X1sa34b43w50JVHZnku0muTXJMkhuSvHXFzXaN\nvvfCJO+rqjcluTLJ25M8P8neJN/cxNO+PskfsxT1VUm+XFU1+t43kvx69L1PJ7lo3AMMw/BokrOT\n3DcMw9GjP/eNvn1ukpuSPDvJ19c6kGEYLkxyT5JzRo9x1bJvn57kJUnenOSyqnrpJv6OHIKXYPPh\nDVn6f/H5YWnzxM1V9csVt/lvksuHYXgiSarq/CRfGYbhN6PPP5rkoao6eYPPuXcYhi+N7vvVJF9I\n8rzRD6HXJjlz9Fw/q6pbJvg77RmG4Xujjx///8+UTbtiGIbHk9xZVXcmeWWS30/6YCyx4s+H45Pc\nOxy8Y+ovK26zfxiGf6+4z94DnwzD8EiSfyQ5YYPP+cCy+z42+vDo0eM+NFrND9ibzVt5/JN6YNnH\nj2XpGDlMwp8P9yc5oQ5eFl+w4jYrt1Hel6WX/UmeOt8+Nsm9SQ5E+8xlt9+1iWN5zujxDjhpjdsf\nanvnyq8/us7x2CY6RcKfD3uS/CfJ+6vqiKo6N8nr1rnPDUneVVWvqqpnJPlsktuHYbh7GIb9WfoB\ncEFVPb2q3p3klI0cyDAMe5P8KskVVXVkVZ2e5Jw17vLXJMdW1bPWeejfJnlLVR1TVbuSfGDM47x4\nI8fI4RP+HBiG4ckk5yV5T5KHk1yQ5AdJnljjPj9O8okk38nSKn1Kkncsu8klST6UpZf/L0vyi00c\n0juzNPx7MMnlSb62xnH8IUs/hO6qqocP/MvCGNcluTPJ3Ul+lOTGFd+/MsnHR4/xwU0cKxMob8Qx\nn6rq9iRfHIbhmlkfCzuPFX9OVNUbq2rX6KX+RUlekeSHsz4udib/nDc/XpLkW0mOSnJXkrcNw3D/\nbA+JncpLfWjIS31oaNov9b282AJX79k/60PYtEtPPW7V1568deKr+VjDkWcN6/6HteJDQ8KHhoQP\nDQkfGvLv+EzF2IHk0X876FMDwOmx4kNDwoeGhA8NOcdnboybA1x61sHXfDnn3xpWfGhI+NCQ8KEh\n4UND096Pb3feJi3iTrxpW3nhT/cBoN15wFjCh4aEDw25gIeFt2oOsmLzT2IOsJIVHxoSPjQkfGhI\n+NCQ4R4tTDIATHbuENCKDw0JHxoSPjRkk86csSlnvi3CHMAmHWAs4UNDwoeGhA8NuYAHNmGn/Cow\nKz40JHxoSPjQkHN82GKLMAew4kNDwoeGhA8NCR8asjtvhuzEYy2TDgDtzgPGEj40JHxoyDn+DDnH\n53CNmwMkcY4PrCZ8aEj40JDwoSG782CBjRsQH2LgdxArPjQkfGhI+NCQc/wpcsEO88KKDw0JHxoS\nPjQkfGhI+NCQ8KEh4UNDwoeGhA8NuXJvm7hKj3lmxYeGhA8NCR8aco6/TV7+qVdPdL/fXXbHFh8J\nrGbFh4aEDw0JHxoSPjQ01eHebWefuO5tTtu9bwpHMr8mGQoaCLJZVnxoSPjQkPChoRqG6f3K+tvO\nPnFLnmze5gAbmV3MmjlAH5eeelytdxsrPjQkfGhI+NCQ8KGhhdydN+kwbd6GgtNktyDLWfGhIeFD\nQ8KHhhbyAp7ttJE5wCJcsDNrZgOz4wIeYCzhQ0PCh4aEDw0Z7jEzBoDbw3APGEv40JDwoaGF3KTD\nzuAdhWfHig8NCR8aEj40JHxoyHCPheKdhLaGFR8aEj40JHxoaKqbdJKs+2Te3YZZ2SlzAJt0gLGE\nDw0JHxoSPjQ0d8O9SRkKMivzNhQ03APGEj40JHxoSPjQ0I4Z7k3CQJBZ2c6BoOEeMJbwoSHhQ0Ot\nz/EnZTbANEw6B3COD4wlfGhI+NCQ8KEhw70pMhRkGk7bvc9wD1hN+NCQ8KEhv0JrAtfdvO4p1FgX\n7t78iMNcgO1gxYeGhA8NCR8aEj40ZLg3RZMMBScZCCaGgqzNig8NCR8aEj40ZJPOOia9WGfWLjzP\nxUJd2aQDjCV8aEj40JDwoSEX8OxQLhZiLVZ8aEj40JDwoSHn+Dxlmu8slJgNzJIVHxoSPjQkfGhI\n+NCQ4R6HbTuHggaA28OKDw0JHxoSPjTkHJ+Z2dBs4JKDP93IOwuZC6zPig8NCR8aEj40JHxoyHCP\nhbKRgaDdguuz4kNDwoeGhA8N+RVaKyzqr8zi8E3ya8eS2c4GTtu9b9yX/QotYDXhQ0PCh4aEDw25\ngAdGpvn24rO+WMiKDw0JHxoSPjQkfGjIcA8O0yRDwVm/tbgVHxoSPjQkfGjI7rwV7M5jnm1wB6Hd\necBqwoeGhA8NCR8acgEPLJCVw+dJ3y7Mig8NCR8aEj405BwfFsik5/QrWfGhIeFDQ8KHhoQPDbUe\n7tmJx6K7es/+VV+79NTj1r2fFR8aEj40JHxoSPjQkPChIeFDQ8KHhoQPDQkfGhI+NCR8aEj40JDw\noSHhQ0PCh4aEDw0JHxpq/Q48MM+26q20x7HiQ0PCh4aEDw0JHxoSPjQkfGhI+NCQ8KGhGobtu0gA\nmE9WfGhI+NCQ8KEh4UNDwoeGhA8NCR8aEj40JHxoSPjQkPChIeFDQ8KHhoQPDQkfGhI+NCR8aEj4\n0JDwoSHhQ0PCh4aEDw0JHxr6H9R08cd1WwguAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe307dd86d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig,ax=plt.subplots()\n",
    "plot_class(ax, 10969, folder=\"ground_truth\", title=\"ground truth\",cmap=\"tum\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53dee7a5daad429393f2fb587885b602",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<function __main__.plot_tile>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tile_slider = widgets.IntSlider(\n",
    "    value=0,\n",
    "    min=0,\n",
    "    max=len(tiles)-1,\n",
    "    step=1,\n",
    "    description='Tile:'\n",
    ")\n",
    "interact(plot_tile,tileidx=tile_slider)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:MTLCC]",
   "language": "python",
   "name": "conda-env-MTLCC-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
